﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Diagnostics;
using System.Drawing;
using System.IO;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using System.Windows.Forms;
using Common.Utilities;
using Common.Utilities.Dispatcher;
using WindowsUserControl.Dialogs;
using WindowsUserControl.UIUpdateWrapper;
using ZedGraph;

namespace Workflows.Components.TextMining.Collapse
{
    /// <summary>
    /// 
    /// </summary>
    public partial class EntityCollapseEditor : Form, IComponentEditor
    {
        private object _OwnerActivityNode;
        private EntityCollapseComponent _EntityCollapseComponent;
        private string _CurrentSelectedFieldName;
        private Dictionary<string, string> _SimpleTypeName_AssemblyName_Mappings;
        private Dictionary<string, string> _SimpleTypeName_FullTypeName_Mappings;

        #region test results
        private List<int> _UnderCollapsedIDs;
        private List<int> _OverCollapsedIDs;
        #endregion

        /// <summary>
        /// 
        /// </summary>
        public EntityCollapseEditor()
        {
            InitializeComponent();
        }

        private void EntityCollapseEditor_Load(object sender, EventArgs e)
        {
            Assembly assembly = this.GetType().Assembly;
            Type[] types = assembly.GetTypes();
            Type comparerInterfaceType = typeof (IAttributeComparer);
            this._SimpleTypeName_AssemblyName_Mappings = new Dictionary<string, string>();
            this._SimpleTypeName_FullTypeName_Mappings=new Dictionary<string, string>();
            foreach (Type type in types)
            {
                bool add = true;
                if (comparerInterfaceType != null)
                {
                    if (type.GetInterface(comparerInterfaceType.Name) == null)
                    {
                        add = false;
                    }
                }
                if (add)
                {
                    this._SimpleTypeName_AssemblyName_Mappings.Add(type.Name, Path.GetFileName(assembly.Location));
                    this._SimpleTypeName_FullTypeName_Mappings.Add(type.Name, type.FullName);
                }
            }

            Type compensateType = typeof (ICompensater);
            this.cbo_CompositeComparers.Items.Clear();
            this.cbo_CompositeComparers.Items.Add("...");
            foreach(Type type in types)
            {
                if(type.GetInterface(compensateType.Name)!=null)
                {
                    this.cbo_CompositeComparers.Items.Add(type.FullName);
                }
            }

            this.PopulateComparers();
        }

        private void PopulateComparers()
        {
            this.cbo_Comparers.Items.Clear();
            Assembly assembly = this.GetType().Assembly;
            Type[] types = assembly.GetTypes();
            foreach(Type type in types)
            {
                if(type.GetInterface(typeof(IAttributeComparer).FullName)!=null)
                {
                    this.cbo_Comparers.Items.Add(type.FullName);
                }
            }
        }

        private void btn_Refresh_Click(object sender, EventArgs e)
        {
            if(!string.IsNullOrEmpty(this.txt_ConnStr.Text))
            {
                this.PopulateSchemas(this.txt_ConnStr.Text);
                if (string.IsNullOrEmpty(this._EntityCollapseComponent.SchemaName))
                    this._EntityCollapseComponent.SchemaName = "dbo";
                this.SelectComboBoxValue(this.cbo_Schemas, this._EntityCollapseComponent.SchemaName);
                this.PopulateTableNames(this.txt_ConnStr.Text,this._EntityCollapseComponent.SchemaName);
                this.SelectComboBoxValue(this.cbo_Tbls,this._EntityCollapseComponent.TableName);
            }
        }

        private void cbo_Tbls_SelectedIndexChanged(object sender, EventArgs e)
        {
            if((!string.IsNullOrEmpty(this.Text)) && this.cbo_Tbls.SelectedIndex>=0)
            {
                int recordCountForTable =
                    DbUtil.GetTableRecordCount(
                        DataProviderType.MSSQL, this.txt_ConnStr.Text, this.cbo_Schemas.Text, this.cbo_Tbls.Text);
                if (recordCountForTable > 0)
                {
                    if (this.trk_SampleSize.Value > recordCountForTable)
                    {
                        this.trk_SampleSize.Value = recordCountForTable;
                        this.lbl_SampleSize.Text = string.Format("Test sample size: {0}", recordCountForTable);
                    }
                    this.trk_SampleSize.Maximum = recordCountForTable;
                    int interval = recordCountForTable/20;
                    this.trk_SampleSize.TickFrequency = (int) (interval/1000)*1000;
                }

                TableColumn[] cols = DbUtil.GetTableColumns(DataProviderType.MSSQL, this.txt_ConnStr.Text,
                                                            this.cbo_Schemas.Text,
                                                            this.cbo_Tbls.Text);
                this.cbo_PkField.Items.Clear();
                this.cbo_PkField.SelectedIndex = -1;
                this.cbo_PkField.Text = string.Empty;
                this.dv_Fields.Rows.Clear();
                this.clb_AddFilters.Items.Clear();
                this.lb_OrFilters.Items.Clear();
                this.dv_Encoders.Rows.Clear();
                this.clb_AvaiFields.Items.Clear();
                this.lb_SwappedFields.Items.Clear();
                this.clb_AvailableFields3.Items.Clear();
                this.lb_CompositeFields.Items.Clear();

                if (cols != null && cols.Length > 0)
                {
                    foreach (TableColumn col in cols)
                    {
                        this.cbo_PkField.Items.Add(col.ColumnName);
                        this.clb_AddFilters.Items.Add(col.ColumnName);
                        this.clb_AvaiFields.Items.Add(col.ColumnName);
                        this.clb_AvailableFields3.Items.Add(col.ColumnName);

                        DataGridViewRow dr = new DataGridViewRow();
                        DataGridViewTextBoxCell cellFieldName = new DataGridViewTextBoxCell();
                        cellFieldName.Value = col.ColumnName;
                        dr.Cells.Add(cellFieldName);
                        DataGridViewCheckBoxCell cellInclude = new DataGridViewCheckBoxCell();
                        dr.Cells.Add(cellInclude);
                        DataGridViewComboBoxCell cellType = new DataGridViewComboBoxCell();
                        cellType.Items.Add("Descriptive field");
                        cellType.Items.Add("Associative field");
                        dr.Cells.Add(cellType);
                        DataGridViewTextBoxCell cellWeight = new DataGridViewTextBoxCell();
                        cellWeight.Value = 0.00;
                        dr.Cells.Add(cellWeight);
                        DataGridViewTextBoxCell cellPenalty = new DataGridViewTextBoxCell();
                        cellPenalty.Value = 0.00;
                        dr.Cells.Add(cellPenalty);
                        DataGridViewCheckBoxCell cellUneven = new DataGridViewCheckBoxCell();
                        dr.Cells.Add(cellUneven);
                        DataGridViewComboBoxCell cellComparer = new DataGridViewComboBoxCell();
                        List<string> compareTypeNames=new List<string>();
                        foreach(string simpleTypeName in this._SimpleTypeName_AssemblyName_Mappings.Keys)
                        {
                            compareTypeNames.Add(simpleTypeName);
                        }
                        compareTypeNames.Sort();
                        cellComparer.Items.Clear();
                        cellComparer.Items.Add("...");
                        foreach (string compareTypeName in compareTypeNames)
                        {
                            cellComparer.Items.Add(compareTypeName);
                        }
                        cellComparer.Items.Add("Choose...");
                        cellComparer.Value = "...";
                        dr.Cells.Add(cellComparer);
                        this.dv_Fields.Rows.Add(dr);
                    }
                }

                if(!string.IsNullOrEmpty(this._EntityCollapseComponent.IDFieldName))
                {
                    for(int i=0;i<this.cbo_PkField.Items.Count;i++)
                    {
                        if(this.cbo_PkField.Items[i].ToString()==this._EntityCollapseComponent.IDFieldName)
                        {
                            this.cbo_PkField.SelectedIndex = i;
                            break;
                        }
                    }
                }
            }
        }

        private void btn_Test_Click(object sender, EventArgs e)
        {
            TestResult result = this.Test();
            // MessageBox.Show(result.Message, result.Succeed.ToString());
        }

        #region IComponentEditor Members

        public event UpdateComponentSettings UpdateComponentSettingEvent;

        public object OwnerActivityNode
        {
            get { return _OwnerActivityNode; }
            set { _OwnerActivityNode = value; }
        }

        public Type WorkflowComponentType
        {
            get { return typeof(EntityCollapseComponent); }
        }

        private void SelectComboBoxValue(ComboBox comboBox, string value)
        {
            if(!string.IsNullOrEmpty(value) && comboBox.Items.Count>0)
            {
                for(int i=0;i< comboBox.Items.Count;i++)
                {
                    if(comboBox.Items[i].ToString().ToLower()==value.ToLower())
                    {
                        comboBox.SelectedIndex = i;
                        break;
                    }
                }
            }
        }

        public void LoadSettings(GlobalVariable[] globalVars, WorkflowComponentBase[] fromComponents,
                                 object activityNode, WorkflowComponentBase wfComponent)
        {
            this._OwnerActivityNode = activityNode;
            this._EntityCollapseComponent = (EntityCollapseComponent) wfComponent;

            this.txt_ConnStr.Text = this._EntityCollapseComponent.ConnStr;
            this.PopulateSchemas(this._EntityCollapseComponent.ConnStr);
            this.SelectComboBoxValue(this.cbo_Schemas, this._EntityCollapseComponent.SchemaName);
            if(string.IsNullOrEmpty(this._EntityCollapseComponent.SchemaName))
            {
                this._EntityCollapseComponent.SchemaName = "dbo";
            }
            this.PopulateTableNames(this._EntityCollapseComponent.ConnStr, this._EntityCollapseComponent.SchemaName);
            this.SelectComboBoxValue(this.cbo_Tbls, this._EntityCollapseComponent.TableName);
            if(this.cbo_DistTable.Items.Count>0 && (!string.IsNullOrEmpty(this._EntityCollapseComponent.FieldDistributionSourceTableName)))
            {
                for(int i=0;i<this.cbo_DistTable.Items.Count;i++)
                {
                    if(this.cbo_DistTable.Items[i].ToString()==this._EntityCollapseComponent.FieldDistributionSourceTableName)
                    {
                        this.cbo_DistTable.SelectedIndex = i;
                        break;
                    }
                }
            }
            this.num_CollapseThreshold.Value = (decimal) this._EntityCollapseComponent.CollapseThreshold;
            this.num_AssocThreshold.Value = (decimal) this._EntityCollapseComponent.AssociateThreshold;
            
            if(this._EntityCollapseComponent.FieldComparers !=null && this._EntityCollapseComponent.FieldComparers.Count>0)
            {
                foreach(string fieldName in this._EntityCollapseComponent.FieldComparers.Keys)
                {
                    FieldComparerPointer fieldComparer = this._EntityCollapseComponent.FieldComparers[fieldName];
                    string typeName = fieldComparer.TypeFullName;
                    string simpleTypeName = string.Empty;
                    foreach(string key in this._SimpleTypeName_FullTypeName_Mappings.Keys)
                    {
                        if (this._SimpleTypeName_FullTypeName_Mappings[key] == typeName)
                        {
                            simpleTypeName = key;
                            break;
                        }
                    }
                    if(string.IsNullOrEmpty(simpleTypeName))
                    {
                        simpleTypeName = typeName.Substring(typeName.LastIndexOf(".") + 1);
                        this._SimpleTypeName_AssemblyName_Mappings.Add(simpleTypeName,fieldComparer.AssemblyFileName);
                        this._SimpleTypeName_FullTypeName_Mappings.Add(simpleTypeName,typeName);
                    }
                }
            }
            List<string> compareTypeNames = new List<string>();
            foreach (string compareTypeName in this._SimpleTypeName_AssemblyName_Mappings.Keys)
            {
                compareTypeNames.Add(compareTypeName);
            }
            compareTypeNames.Sort();

            foreach(DataGridViewRow dr in this.dv_Fields.Rows)
            {
                if(!dr.IsNewRow && dr.Cells[0].Value !=null)
                {
                    string fieldName = dr.Cells[0].Value.ToString();
                    DataGridViewCheckBoxCell includeCell = (DataGridViewCheckBoxCell)dr.Cells[1];
                    includeCell.Value = false;
                    DataGridViewComboBoxCell attTypeCell = (DataGridViewComboBoxCell) dr.Cells[2];
                    DataGridViewTextBoxCell weightCell = (DataGridViewTextBoxCell) dr.Cells[3];
                    weightCell.Value = 0.00;
                    DataGridViewTextBoxCell cellPenalty = (DataGridViewTextBoxCell) dr.Cells[4];
                    cellPenalty.Value = 0.00;
                    DataGridViewCheckBoxCell unevenCell = (DataGridViewCheckBoxCell) dr.Cells[5];
                    unevenCell.Value = false;
                    DataGridViewComboBoxCell cellComparer = (DataGridViewComboBoxCell) dr.Cells[6];
                    cellComparer.Items.Clear();
                    cellComparer.Items.Add("...");
                    foreach (string compareTypeName in compareTypeNames)
                    {
                        cellComparer.Items.Add(compareTypeName);
                    }
                    cellComparer.Items.Add("Choose...");
                    cellComparer.Value = "...";
                    if(this._EntityCollapseComponent.FieldComparers!=null && this._EntityCollapseComponent.FieldComparers.ContainsKey(fieldName))
                    {
                        FieldComparerPointer fieldComparer = this._EntityCollapseComponent.FieldComparers[fieldName];
                        foreach(string simpleTypeName in compareTypeNames)
                        {
                            if(this._SimpleTypeName_FullTypeName_Mappings[simpleTypeName]==fieldComparer.TypeFullName)
                            {
                                cellComparer.Value = simpleTypeName;
                                break;
                            }
                        }
                    }

                    if (this._EntityCollapseComponent.PenaltiesWhenNotMatched == null)
                        this._EntityCollapseComponent.PenaltiesWhenNotMatched=new Dictionary<string, double>();

                    if(this._EntityCollapseComponent.DescriptiveAttributes.ContainsKey(fieldName))
                    {
                        attTypeCell.Value = "Descriptive field";
                        includeCell.Value = true;
                        weightCell.Value = this._EntityCollapseComponent.DescriptiveAttributes[fieldName];
                        if (this._EntityCollapseComponent.PenaltiesWhenNotMatched.ContainsKey(fieldName))
                            cellPenalty.Value = this._EntityCollapseComponent.PenaltiesWhenNotMatched[fieldName];
                    }
                    if(this._EntityCollapseComponent.AssociativeAttributes.ContainsKey(fieldName))
                    {
                        attTypeCell.Value = "Associative field";
                        includeCell.Value = true;
                        weightCell.Value = this._EntityCollapseComponent.AssociativeAttributes[fieldName];
                        if (this._EntityCollapseComponent.PenaltiesWhenNotMatched.ContainsKey(fieldName))
                            cellPenalty.Value = this._EntityCollapseComponent.PenaltiesWhenNotMatched[fieldName];
                    }

                    
                    if(this._EntityCollapseComponent.UnevenlyDistributedFields!=null && 
                        this._EntityCollapseComponent.UnevenlyDistributedFields.Contains(fieldName))
                    {
                        unevenCell.Value = true;
                    }
                }
            }

            this.lb_OrFilters.Items.Clear();
            this.dv_Encoders.Rows.Clear();
            List<string> distFilterFields=new List<string>();
            if(this._EntityCollapseComponent.OrFilters !=null && this._EntityCollapseComponent.OrFilters.Count>0)
            {
                for(int i=0;i<this._EntityCollapseComponent.OrFilters.Count;i++)
                {
                    List<string> addFilters = this._EntityCollapseComponent.OrFilters[i];
                    string filterString = string.Empty;
                    for(int k=0;k<addFilters.Count;k++)
                    {
                        filterString += addFilters[k];
                        if(k<addFilters.Count -1)
                        {
                            filterString += " and ";
                        }

                        if(!distFilterFields.Contains(addFilters[k]))
                            distFilterFields.Add(addFilters[k]);
                    }
                    this.lb_OrFilters.Items.Add(filterString);
                }
            }
            Assembly assembly = this.GetType().Assembly;
            List<string> encoderNames=new List<string>();
            Type[] types = assembly.GetTypes();
            foreach(Type type in types)
            {
                if(type.GetInterface(typeof(IFieldIndexer).FullName)!=null)
                    encoderNames.Add(type.FullName);
            }
            if(distFilterFields.Count>0)
            {
                foreach(string filterField in distFilterFields)
                {
                    DataGridViewRow dr = new DataGridViewRow();
                    DataGridViewTextBoxCell cellFilter=new DataGridViewTextBoxCell();
                    cellFilter.Value = filterField;
                    dr.Cells.Add(cellFilter);
                    DataGridViewComboBoxCell cellEncoders=new DataGridViewComboBoxCell();
                    if(encoderNames !=null && encoderNames.Count>0)
                    {
                        foreach (string encoderName in encoderNames)
                            cellEncoders.Items.Add(encoderName);
                    }
                    if(this._EntityCollapseComponent.FieldEncoders !=null &&
                        this._EntityCollapseComponent.FieldEncoders.ContainsKey(filterField))
                    {
                        cellEncoders.Value = this._EntityCollapseComponent.FieldEncoders[filterField];
                    }
                    dr.Cells.Add(cellEncoders);
                    this.dv_Encoders.Rows.Add(dr);
                }
            }

            this.lb_SwappedFields.Items.Clear();
            if(this._EntityCollapseComponent.SwappedFieldsCollection != null && this._EntityCollapseComponent.SwappedFieldsCollection.Count>0)
            {
                for(int i=0;i<this._EntityCollapseComponent.SwappedFieldsCollection.Count;i++)
                {
                    List<string> swapFieldNames = this._EntityCollapseComponent.SwappedFieldsCollection[i];
                    string swapStr = string.Empty;
                    for (int k = 0; k < swapFieldNames.Count; k++)
                    {
                        swapStr += swapFieldNames[k];
                        if (k < swapFieldNames.Count - 1)
                        {
                            swapStr += " and ";
                        }
                    }
                    this.lb_SwappedFields.Items.Add(swapStr);
                }
            }

            this.txt_OutputTableName.Text = this._EntityCollapseComponent.OutputTableName;
            this.txt_CollapseTable.Text = this._EntityCollapseComponent.CollapsedTableName;
            if(string.IsNullOrEmpty(this._EntityCollapseComponent.UnderCollapsedFilePath))
            {
                this.ck_UnderCollapsed.Checked = false;
                this.txt_UnderCollapsedFilePath.Text = string.Empty;
            }
            else
            {
                this.ck_UnderCollapsed.Checked = true;
                this.txt_UnderCollapsedFilePath.Text = this._EntityCollapseComponent.UnderCollapsedFilePath;
            }
            if(string.IsNullOrEmpty(this._EntityCollapseComponent.OverCollapsedFilePath))
            {
                this.ck_OverCollapsed.Checked = false;
                this.txt_OverCollapsedFilePath.Text = string.Empty;
            }
            else
            {
                this.ck_OverCollapsed.Checked = true;
                this.txt_OverCollapsedFilePath.Text = this._EntityCollapseComponent.OverCollapsedFilePath;
            }
            if(string.IsNullOrEmpty(this._EntityCollapseComponent.FinalResultFilePath))
            {
                this.ck_SaveFinalResult.Checked = false;
                this.txt_FinalResultFilePath.Text = string.Empty;
            }
            else
            {
                this.ck_SaveFinalResult.Checked = true;
                this.txt_FinalResultFilePath.Text = this._EntityCollapseComponent.FinalResultFilePath;
            }

            this.dv_CompositeComparer.Rows.Clear();
            if (this._EntityCollapseComponent.Compensaters != null && this._EntityCollapseComponent.Compensaters.Count > 0)
            {
                foreach (CompensaterPointer compensater in this._EntityCollapseComponent.Compensaters)
                {
                    this.AddCompensater(compensater);
                }
            }
        }

        public void SaveSettings()
        {
            this._EntityCollapseComponent.ConnStr = this.txt_ConnStr.Text;
            this._EntityCollapseComponent.SchemaName = this.cbo_Schemas.Text;
            this._EntityCollapseComponent.TableName = this.cbo_Tbls.Text;
            this._EntityCollapseComponent.FieldDistributionSourceTableName = this.cbo_DistTable.Text;
            this._EntityCollapseComponent.EntityName = this.cbo_Tbls.Text;
            this._EntityCollapseComponent.IDFieldName = this.cbo_PkField.Text;
            this._EntityCollapseComponent.CollapseThreshold = (double) this.num_CollapseThreshold.Value;
            this._EntityCollapseComponent.AssociateThreshold = (double) this.num_AssocThreshold.Value;

            this._EntityCollapseComponent.DescriptiveAttributes=new Dictionary<string, double>();
            this._EntityCollapseComponent.AssociativeAttributes=new Dictionary<string, double>();
            this._EntityCollapseComponent.UnevenlyDistributedFields=new List<string>();
            this._EntityCollapseComponent.PenaltiesWhenNotMatched=new Dictionary<string, double>();
            this._EntityCollapseComponent.FieldComparers = new Dictionary<string, FieldComparerPointer>();

            foreach(DataGridViewRow dr in this.dv_Fields.Rows)
            {
                if(!dr.IsNewRow && dr.Cells[0].Value !=null)
                {
                    string fieldName = dr.Cells[0].Value.ToString();
                    bool include = dr.Cells[1].Value == null ? false : (bool)dr.Cells[1].Value;
                    string attributeType = dr.Cells[2].Value == null ? "Descriptive field" : dr.Cells[2].Value.ToString();
                    double weight = dr.Cells[3].Value == null || dr.Cells[3].Value == DBNull.Value
                                        ? 0.0
                                        : double.Parse(dr.Cells[3].Value.ToString());
                    double penalty = dr.Cells[4].Value == null || dr.Cells[4].Value == DBNull.Value
                                         ? 0.0
                                         : double.Parse(dr.Cells[4].Value.ToString());
                    bool uneven = dr.Cells[5].Value == null ? false : (bool) dr.Cells[5].Value;
                    string typeName = dr.Cells[6].Value == null ? "..." : dr.Cells[6].Value.ToString();
                    if(include)
                    {
                        if (attributeType == "Descriptive field")
                        {
                            this._EntityCollapseComponent.DescriptiveAttributes.Add(fieldName,weight);
                        }    
                        else
                        {
                            this._EntityCollapseComponent.AssociativeAttributes.Add(fieldName,weight);
                        }
                        if(this._EntityCollapseComponent.PenaltiesWhenNotMatched!=null && 
                            this._EntityCollapseComponent.PenaltiesWhenNotMatched.ContainsKey(fieldName))
                        {
                           this._EntityCollapseComponent.PenaltiesWhenNotMatched[fieldName]=penalty; 
                        }
                        else
                        {
                            this._EntityCollapseComponent.PenaltiesWhenNotMatched.Add(fieldName,penalty);
                        }
                        if(uneven)
                        {
                            this._EntityCollapseComponent.UnevenlyDistributedFields.Add(fieldName);
                        }

                        if(typeName !="..." && typeName!="Choose..." && this._SimpleTypeName_AssemblyName_Mappings.ContainsKey(typeName))
                        {
                            FieldComparerPointer fieldComparer=new FieldComparerPointer();
                            fieldComparer.AssemblyFileName = this._SimpleTypeName_AssemblyName_Mappings[typeName];
                            fieldComparer.TypeFullName = this._SimpleTypeName_FullTypeName_Mappings[typeName];
                            this._EntityCollapseComponent.FieldComparers.Add(fieldName, fieldComparer);
                        }
                    }
                }
            }

            if (this._EntityCollapseComponent.FieldComparers !=null && this._EntityCollapseComponent.FieldComparers.Count>0)
            {
                List<string> compareFieldNamesNotIncluded = new List<string>();
                foreach(string fieldName in this._EntityCollapseComponent.FieldComparers.Keys)
                {
                    if(!this._EntityCollapseComponent.DescriptiveAttributes.ContainsKey(fieldName) && 
                        !this._EntityCollapseComponent.AssociativeAttributes.ContainsKey(fieldName))
                    {
                        compareFieldNamesNotIncluded.Add(fieldName);
                    }
                }
                if (compareFieldNamesNotIncluded.Count > 0)
                {
                    foreach (string excludedFieldName in compareFieldNamesNotIncluded)
                    {
                        this._EntityCollapseComponent.FieldComparers.Remove(excludedFieldName);
                    }
                }
            }
            this._EntityCollapseComponent.OrFilters=new List<List<string>>();
            Regex split=new Regex(" and ", RegexOptions.IgnoreCase);
            if(this.lb_OrFilters.Items.Count>0)
            {
                for(int i=0;i<this.lb_OrFilters.Items.Count;i++)
                {
                    List<string> addFilters = new List<string>();
                    string addFilter = this.lb_OrFilters.Items[i].ToString();
                    if(Regex.Match(addFilter, " and ", RegexOptions.IgnoreCase).Success)
                    {
                        string[] fieldsForAddFilter = split.Split(addFilter);
                        addFilters.AddRange(fieldsForAddFilter);
                    }
                    else
                    {
                        addFilters.Add(addFilter);
                    }
                    this._EntityCollapseComponent.OrFilters.Add(addFilters);
                }
            }

            this._EntityCollapseComponent.FieldEncoders=new Dictionary<string, string>();
            foreach(DataGridViewRow dr in this.dv_Encoders.Rows)
            {
                if(!dr.IsNewRow && dr.Cells[0].Value!=null && dr.Cells[1].Value !=null)
                {
                    string fieldName = dr.Cells[0].Value.ToString();
                    string encoder = dr.Cells[1].Value.ToString();
                    this._EntityCollapseComponent.FieldEncoders.Add(fieldName, encoder);
                }
            }

            this._EntityCollapseComponent.SwappedFieldsCollection=new List<List<string>>();
            if(this.lb_SwappedFields.Items.Count>0)
            {
                for (int i = 0; i < this.lb_SwappedFields.Items.Count; i++)
                {
                    List<string> swapFieldNames = new List<string>();
                    string swapStr = this.lb_SwappedFields.Items[i].ToString();
                    if (Regex.Match(swapStr, " and ", RegexOptions.IgnoreCase).Success)
                    {
                        string[] fieldsForSwap = split.Split(swapStr);
                        swapFieldNames.AddRange(fieldsForSwap);
                    }
                    else
                    {
                        swapFieldNames.Add(swapStr);
                    }
                    this._EntityCollapseComponent.SwappedFieldsCollection.Add(swapFieldNames);
                }
            }

            this._EntityCollapseComponent.OutputTableName = this.txt_OutputTableName.Text;
            this._EntityCollapseComponent.CollapsedTableName = this.txt_CollapseTable.Text;

            if(this.ck_UnderCollapsed.Checked && (!string.IsNullOrEmpty(this.txt_UnderCollapsedFilePath.Text)))
            {
                this._EntityCollapseComponent.UnderCollapsedFilePath = this.txt_UnderCollapsedFilePath.Text;
            }
            else
            {
                this._EntityCollapseComponent.UnderCollapsedFilePath = string.Empty;
            }
            if(this.ck_OverCollapsed.Checked && (!string.IsNullOrEmpty(this.txt_OverCollapsedFilePath.Text)))
            {
                this._EntityCollapseComponent.OverCollapsedFilePath = this.txt_OverCollapsedFilePath.Text;
            }
            else
            {
                this._EntityCollapseComponent.OverCollapsedFilePath = string.Empty;
            }
            if(this.ck_SaveFinalResult.Checked && (!string.IsNullOrEmpty(this.txt_FinalResultFilePath.Text)))
            {
                this._EntityCollapseComponent.FinalResultFilePath = this.txt_FinalResultFilePath.Text;
            }
            else
            {
                this._EntityCollapseComponent.FinalResultFilePath = string.Empty;
            }

            this._EntityCollapseComponent.Compensaters = this.GetCompensaters();
        }

        public TestResult Test()
        {
            this.SaveSettings();
            EntityCollapseInstruction instruction =
                new EntityCollapseInstruction(this._EntityCollapseComponent.ComponentID, Guid.Empty,
                                              this._EntityCollapseComponent.EntityName,
                                              this._EntityCollapseComponent.ConnStr,
                                              this._EntityCollapseComponent.SchemaName,
                                              this._EntityCollapseComponent.TableName,
                                              this._EntityCollapseComponent.IDFieldName,
                                              this._EntityCollapseComponent.DescriptiveAttributes,
                                              this._EntityCollapseComponent.AssociativeAttributes,
                                              this._EntityCollapseComponent.PenaltiesWhenNotMatched,
                                              this._EntityCollapseComponent.FieldComparers,
                                              this._EntityCollapseComponent.UnevenlyDistributedFields,
                                              this._EntityCollapseComponent.OrFilters,
                                              this._EntityCollapseComponent.FieldEncoders,
                                              this._EntityCollapseComponent.CollapseThreshold,
                                              this._EntityCollapseComponent.OutputTableName, 
                                              this._EntityCollapseComponent.CollapsedTableName,
                                              this._EntityCollapseComponent.FieldDistributionSourceTableName,
                                              this._EntityCollapseComponent.SwappedFieldsCollection,
                                              this._EntityCollapseComponent.AssociateThreshold,
                                              this._EntityCollapseComponent.OverCollapsedFilePath,
                                              this._EntityCollapseComponent.UnderCollapsedFilePath,
                                              this._EntityCollapseComponent.FinalResultFilePath, 
                                              this._EntityCollapseComponent.Compensaters);
            EntityCollapseWorker worker=new EntityCollapseWorker();
            worker.SetInstruction(Guid.NewGuid(), instruction, string.Empty, this._EntityCollapseComponent.ComponentName);
            worker.WorkerErredEvent += new WorkerHasError(worker_WorkerErredEvent);
            worker.WorkerFinishedEvent += new WorkerFinished(worker_WorkerFinishedEvent);
            worker.WorkerStatusChangedEvent += new WorkerStatusChanged(worker_WorkerStatusChangedEvent);
            worker.OnCollapseProgressing += new CollapseProgressing(worker_OnCollapseProgressing);

            worker.TestMode = true;
            worker.TableSampleSize = this.trk_SampleSize.Value;
            worker.OnFieldContributionsUpdated += new FieldsContributionUpdated(worker_OnFieldContributionsUpdated);
            worker.OnUnderCollapsedEntitiesUpdated += new UnderCollapsedEntitiesUpdated(worker_OnUnderCollapsedEntitiesUpdated);
            worker.OnUnderCollapsedEntitiesPopulated += new UnderCollapsedEntitiesPopulated(worker_OnUnderCollapsedEntitiesPopulated);
            worker.OnOverCollapsedEntitiesUpdated += new OverCollapsedEntitiesUpdated(worker_OnOverCollapsedEntitiesUpdated);
            worker.OnOverCollapsedEntitiesPopulated += new OverCollapsedEntitiesPopulated(worker_OnOverCollapsedEntitiesPopulated);
            worker.DoWorkAsync();
            
            return new TestResult(true, "Started!");
        }

        void ExcelTableWithGroupsUtil_OnPopulatingRow(int rowNumber, int totalRows)
        {
            throw new NotImplementedException();
        }

        void worker_OnOverCollapsedEntitiesPopulated(DataTable dtOverCollapsedEntities, string collapsedEntitiesTableName, string collapsedIDFieldName)
        {
            // col headers
            List<string> colHeaders = new List<string>();
            colHeaders.Add(collapsedIDFieldName);
            colHeaders.Add(this._EntityCollapseComponent.IDFieldName);
            colHeaders.Add("SimilarityScore");
            foreach (string colName in this._EntityCollapseComponent.DescriptiveAttributes.Keys)
            {
                colHeaders.Add(colName);
            }
            foreach (string colName in this._EntityCollapseComponent.AssociativeAttributes.Keys)
            {
                colHeaders.Add(colName);
            }

            this._OverCollapsedIDs=new List<int>();
            List<int> entityIDs=new List<int>();

            if (dtOverCollapsedEntities != null && dtOverCollapsedEntities.Rows.Count > 0)
            {
                foreach (DataRow dr in dtOverCollapsedEntities.Rows)
                {
                    
                    int collapseID1 = (int) dr[collapsedIDFieldName];
                    if (!this._OverCollapsedIDs.Contains(collapseID1))
                    {
                        this._OverCollapsedIDs.Add(collapseID1);
                    }
                    int collapseID2 = (int) dr[collapsedIDFieldName + "_Compare"];
                    if (!this._OverCollapsedIDs.Contains(collapseID2))
                    {
                        this._OverCollapsedIDs.Add(collapseID2);
                    }
                }
            }
            DataTable dtMaster = DbUtil.GrabDataByKeyValues(
                this._EntityCollapseComponent.ConnStr,
                collapsedEntitiesTableName,
                collapsedIDFieldName,
                this._OverCollapsedIDs);
            DataTable dtMappings = DbUtil.GrabDataByKeyValues(
                this._EntityCollapseComponent.ConnStr,
                this._EntityCollapseComponent.OutputTableName,
                collapsedIDFieldName,
                this._OverCollapsedIDs);
            Dictionary<int, int> entityID_CollapseID_Mappings = new Dictionary<int, int>();
            foreach (DataRow dr in dtMappings.Rows)
            {
                int entityID = (int)dr[this._EntityCollapseComponent.IDFieldName];
                int collapseID = (int)dr[collapsedIDFieldName];
                if (!entityIDs.Contains(entityID))
                {
                    entityIDs.Add(entityID);
                    entityID_CollapseID_Mappings.Add(entityID, collapseID);
                }
            }
            string selectColNameSql = "select a." + this._EntityCollapseComponent.IDFieldName + ",";
            foreach (string fieldName in this._EntityCollapseComponent.DescriptiveAttributes.Keys)
            {
                selectColNameSql += "a." + fieldName + ",";
            }
            foreach (string fieldName in this._EntityCollapseComponent.AssociativeAttributes.Keys)
            {
                selectColNameSql += "a." + fieldName + ",";
            }
            string selectSql =
                string.Format(
                    selectColNameSql + " b.SimilarityScore from {0}.{1} a, {2} b where a.{3}=b.{3}",
                    this._EntityCollapseComponent.SchemaName,
                    this._EntityCollapseComponent.TableName,
                    this._EntityCollapseComponent.OutputTableName,
                    this._EntityCollapseComponent.IDFieldName);
            DataTable dtEntities = DbUtil.GrabDataBySqlAndKeyValues(
                this._EntityCollapseComponent.ConnStr,
                selectSql,
                this._EntityCollapseComponent.IDFieldName,
                entityIDs);

            TreeGridViewUpdator tdvWrapper = new TreeGridViewUpdator(
                this.dv_OverCollapsed,
                dtMaster,
                dtEntities,
                collapsedIDFieldName,
                this._EntityCollapseComponent.IDFieldName,
                entityID_CollapseID_Mappings,
                colHeaders);
        }

        void worker_OnUnderCollapsedEntitiesPopulated(
            DataTable dtUnderCollapsedEntities, 
            string collapsedEntitiesTableName, 
            string collapsedIDFieldName)
        {
            // col headers
            List<string> colHeaders = new List<string>();
            colHeaders.Add(collapsedIDFieldName);
            colHeaders.Add(this._EntityCollapseComponent.IDFieldName);
            colHeaders.Add("SimilarityScore");
            foreach (string colName in this._EntityCollapseComponent.DescriptiveAttributes.Keys)
            {
                colHeaders.Add(colName);
            }
            foreach (string colName in this._EntityCollapseComponent.AssociativeAttributes.Keys)
            {
                colHeaders.Add(colName);
            }

            // master-detail assoc
            this._UnderCollapsedIDs = new List<int>();
            if (dtUnderCollapsedEntities != null && dtUnderCollapsedEntities.Rows.Count > 0)
            {
                foreach (DataRow dr in dtUnderCollapsedEntities.Rows)
                {
                    int colllapseID1 = (int)dr[collapsedIDFieldName + "_Compare"];
                    if (!this._UnderCollapsedIDs.Contains(colllapseID1))
                    {
                        this._UnderCollapsedIDs.Add(colllapseID1);
                    }
                    int colllapseID2 = (int)dr[collapsedIDFieldName];
                    if (!this._UnderCollapsedIDs.Contains(colllapseID2))
                    {
                        this._UnderCollapsedIDs.Add(colllapseID2);
                    }
                }
            }
            DataTable dtProfiledEntities = DbUtil.GrabDataByKeyValues(
                this._EntityCollapseComponent.ConnStr,
                collapsedEntitiesTableName,
                collapsedIDFieldName,
                this._UnderCollapsedIDs);
            DataTable dtMappings = DbUtil.GrabDataByKeyValues(
                this._EntityCollapseComponent.ConnStr,
                this._EntityCollapseComponent.OutputTableName,
                collapsedIDFieldName,
                this._UnderCollapsedIDs);
            List<int> entityIDs=new List<int>();
            Dictionary<int,int> entityID_CollapseID_Mappings=new Dictionary<int, int>();
            foreach(DataRow dr in dtMappings.Rows)
            {
                int entityID = (int) dr[this._EntityCollapseComponent.IDFieldName];
                int collapseID = (int)dr[collapsedIDFieldName];
                if (!entityIDs.Contains(entityID))
                {
                    entityIDs.Add(entityID);
                    entityID_CollapseID_Mappings.Add(entityID, collapseID);
                }
            }
            string selectColNameSql = "select a." + this._EntityCollapseComponent.IDFieldName + ",";
            foreach (string fieldName in this._EntityCollapseComponent.DescriptiveAttributes.Keys)
            {
                selectColNameSql += "a." + fieldName + ",";
            }
            foreach (string fieldName in this._EntityCollapseComponent.AssociativeAttributes.Keys)
            {
                selectColNameSql += "a." + fieldName + ",";
            }
            string selectSql =
                string.Format(
                    selectColNameSql + " b.SimilarityScore from {0}.{1} a, {2} b where a.{3}=b.{3}",
                    this._EntityCollapseComponent.SchemaName,
                    this._EntityCollapseComponent.TableName,
                    this._EntityCollapseComponent.OutputTableName,
                    this._EntityCollapseComponent.IDFieldName);
            DataTable dtEntities = DbUtil.GrabDataBySqlAndKeyValues(
                this._EntityCollapseComponent.ConnStr,
                selectSql,
                this._EntityCollapseComponent.IDFieldName,
                entityIDs);

            TreeGridViewUpdator tdvWrapper = new TreeGridViewUpdator(
                this.dv_UnderCollapsed,
                dtProfiledEntities,
                dtEntities,
                collapsedIDFieldName,
                this._EntityCollapseComponent.IDFieldName,
                entityID_CollapseID_Mappings,
                colHeaders);
        }


        void worker_OnUnderCollapsedEntitiesUpdated(DataTable dtEntities)
        {
            DataGridViewUpdator dvWrapper = new DataGridViewUpdator(this.dv_UnderCollapsed, dtEntities);
        }

        void worker_OnOverCollapsedEntitiesUpdated(DataTable dtEntities)
        {
            DataGridViewUpdator dvWrapper = new DataGridViewUpdator(this.dv_OverCollapsed, dtEntities);
        }

        void worker_OnFieldContributionsUpdated(Dictionary<string, DescriptiveAttributeContribution> fieldContributions)
        {
            string title = "field contribution in collapse";
            string xLabel = "fields";
            string yLabel = "count";
            Dictionary<string, List<double>> values = new Dictionary<string, List<double>>();
            foreach(string fieldName in fieldContributions.Keys)
            {
                DescriptiveAttributeContribution fieldContrib = fieldContributions[fieldName];
                List<double> measurements = new List<double>();
                measurements.Add((double) fieldContrib.NumberMatchedInCollapsedEntities);
                measurements.Add((double)fieldContrib.NumberNotMatchedInCollapsedEntities);
                measurements.Add((double)fieldContrib.NumberMatchedInNonCollapsedEntities);
                measurements.Add((double)fieldContrib.NumberNotMatchedInNonCollapsedEntities);
                values.Add(fieldName, measurements);
            }
            try
            {
                if (values.Count > 0)
                {
                    List<string> stackLabels = new List<string>();
                    stackLabels.Add("Match-collapsed");
                    stackLabels.Add("Mismatch-collapsed");
                    stackLabels.Add("Match-non-collapsed");
                    stackLabels.Add("Mismatch-non-collapsed");
                    OverlayedColumnBarZedChartUpdator chartWrapper =
                        new OverlayedColumnBarZedChartUpdator(
                            this.zedGraph,
                            values, stackLabels,
                            title, xLabel,
                            yLabel);
                }
            }
            catch(Exception ex)
            {
                MessageBox.Show(ex.ToString());
            }
        }

        void worker_WorkerFinishedEvent(Guid ticketID, Type workerType, InstructionBase instructionBase, ResultBase resultBase)
        {
            ProgressBarUpdator wrapper1 = new ProgressBarUpdator(this.statusStrip1, this.bar_Progress, 0);
            StatusBarUpdator wrapper2 = new StatusBarUpdator(this.statusStrip1, this.bar_Status, "Finished.");
            // Trace.WriteLine("Finished.");
        }

        void worker_WorkerStatusChangedEvent(Guid ticketID, Type workerType, InstructionBase instructionBase, 
            string workflowName, string componentName, string message, int percent)
        {
            ProgressBarUpdator wrapper1=new ProgressBarUpdator(this.statusStrip1,this.bar_Progress,percent);
            StatusBarUpdator wrapper2=new StatusBarUpdator(this.statusStrip1,this.bar_Status,message);

            // Trace.WriteLine(string.Format("{0}: {1} percent, {2}",DateTime.Now.ToString(),percent,message));
        }

        void worker_OnCollapseProgressing(int entityNotCollapsed, int entityCollapsed, int collapsedEntities)
        {
            Trace.WriteLine(string.Format("{0}: {1} processed, {2} to do, {3} total collapsed entities",
                                          DateTime.Now.ToString(), entityCollapsed, entityNotCollapsed,
                                          collapsedEntities));
        }

        void worker_WorkerErredEvent(Guid ticketID, Type workerType, InstructionBase instructionBase, string error, bool isFatal)
        {
            StatusBarUpdator wrapper2 = new StatusBarUpdator(this.statusStrip1, this.bar_Status, error);
            // Trace.WriteLine(error);
        }

        #endregion

        private void btn_OK_Click(object sender, EventArgs e)
        {
            this.SaveSettings();
            if(this.UpdateComponentSettingEvent!=null)
            {
                this.UpdateComponentSettingEvent(this._OwnerActivityNode, this._EntityCollapseComponent);
            }
            this.Close();
        }

        private void btn_Cancel_Click(object sender, EventArgs e)
        {
            this.Close();
        }

        private void btn_Compare_Click(object sender, EventArgs e)
        {
            this.SaveSettings();
            int entityIDA = int.Parse(this.txt_EntityA.Text);
            int entityIDB = int.Parse(this.txt_EntityB.Text);

            EntityCollapseInstruction instruction =
                new EntityCollapseInstruction(this._EntityCollapseComponent.ComponentID, Guid.Empty,
                                              this._EntityCollapseComponent.EntityName,
                                              this._EntityCollapseComponent.ConnStr,
                                              this._EntityCollapseComponent.SchemaName,
                                              this._EntityCollapseComponent.TableName,
                                              this._EntityCollapseComponent.IDFieldName,
                                              this._EntityCollapseComponent.DescriptiveAttributes,
                                              this._EntityCollapseComponent.AssociativeAttributes,
                                              this._EntityCollapseComponent.PenaltiesWhenNotMatched,
                                              this._EntityCollapseComponent.FieldComparers,
                                              this._EntityCollapseComponent.UnevenlyDistributedFields,
                                              this._EntityCollapseComponent.OrFilters,
                                              this._EntityCollapseComponent.FieldEncoders,
                                              this._EntityCollapseComponent.CollapseThreshold,
                                              this._EntityCollapseComponent.OutputTableName,
                                              this._EntityCollapseComponent.CollapsedTableName,
                                              this._EntityCollapseComponent.FieldDistributionSourceTableName,
                                              this._EntityCollapseComponent.SwappedFieldsCollection,
                                              this._EntityCollapseComponent.AssociateThreshold,
                                              this._EntityCollapseComponent.OverCollapsedFilePath,
                                              this._EntityCollapseComponent.UnderCollapsedFilePath,
                                              this._EntityCollapseComponent.FinalResultFilePath, 
                                              this._EntityCollapseComponent.Compensaters);
            EntityCollapseWorker worker = new EntityCollapseWorker();
            worker.SetInstruction(Guid.NewGuid(), instruction, string.Empty, this._EntityCollapseComponent.ComponentName);
            worker.WorkerErredEvent += new WorkerHasError(worker_WorkerErredEvent);
            worker.WorkerFinishedEvent += new WorkerFinished(worker_WorkerFinishedEvent);
            worker.WorkerStatusChangedEvent += new WorkerStatusChanged(worker_WorkerStatusChangedEvent);
            worker.OnCollapseProgressing += new CollapseProgressing(worker_OnCollapseProgressing);

            List<EntityCompareResult> compareResults = worker.CompareEntity(entityIDA, entityIDB);
            this.dv_Compare.Rows.Clear();
            if(compareResults !=null && compareResults.Count>0)
            {
                foreach(EntityCompareResult result in compareResults)
                {
                    DataGridViewRow dr=new DataGridViewRow();
                    DataGridViewTextBoxCell cellFieldName=new DataGridViewTextBoxCell();
                    cellFieldName.Value = result.FieldName;
                    dr.Cells.Add(cellFieldName);
                    DataGridViewTextBoxCell cellValueA=new DataGridViewTextBoxCell();
                    cellValueA.Value = result.FieldValueA;
                    dr.Cells.Add(cellValueA);
                    DataGridViewTextBoxCell cellValueB=new DataGridViewTextBoxCell();
                    cellValueB.Value = result.FieldValueB;
                    dr.Cells.Add(cellValueB);
                    DataGridViewTextBoxCell cellWeight=new DataGridViewTextBoxCell();
                    cellWeight.Value = result.FieldWeight;
                    dr.Cells.Add(cellWeight);
                    DataGridViewTextBoxCell cellProb=new DataGridViewTextBoxCell();
                    cellProb.Value = result.FieldProbability;
                    dr.Cells.Add(cellProb);
                    DataGridViewTextBoxCell cellScore=new DataGridViewTextBoxCell();
                    cellScore.Value = result.FieldCompareScore;
                    dr.Cells.Add(cellScore);
                    this.dv_Compare.Rows.Add(dr);
                }

                double finalScore = 0;
                foreach(EntityCompareResult result in compareResults)
                {
                    if(instruction.DescriptiveAttributes.ContainsKey(result.FieldName))
                    {
                        finalScore += result.FieldCompareScore;
                    }
                }
                if(finalScore > instruction.AssociateThreshold)
                {
                    foreach (EntityCompareResult result in compareResults)
                    {
                        if (instruction.AssociativeAttributes.ContainsKey(result.FieldName))
                        {
                            finalScore += result.FieldCompareScore;
                        }
                    }
                }
                if (this._EntityCollapseComponent.Compensaters != null && this._EntityCollapseComponent.Compensaters.Count > 0)
                {
                    Dictionary<string, object> entity1Values = new Dictionary<string, object>();
                    Dictionary<string, object> entity2Values = new Dictionary<string, object>();
                    foreach (EntityCompareResult result in compareResults)
                    {
                        entity1Values.Add(result.FieldName, result.FieldValueA);
                        entity2Values.Add(result.FieldName, result.FieldValueB);
                    }
                    //Dictionary<string, double> fieldScores = new Dictionary<string, double>();
                    //foreach (EntityCompareResult result in compareResults)
                    //{
                    //    fieldScores.Add(result.FieldName, result.FieldCompareScore);
                    //}
                    string binFolderPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
                    foreach (CompensaterPointer compensaterStruct in this._EntityCollapseComponent.Compensaters)
                    {
                        string assemblyFilePath = Path.Combine(binFolderPath, compensaterStruct.AssemblyName);
                        Assembly assembly = Assembly.LoadFile(assemblyFilePath);
                        Type compensaterType = assembly.GetType(compensaterStruct.TypeFullName);
                        ICompensater compensater = (ICompensater)Activator.CreateInstance(compensaterType);
                        compensater.CanSwap = compensaterStruct.CanSwap;
                        compensater.CompensateScore = compensaterStruct.CompensateScore;
                        compensater.CompositeFieldNames = compensaterStruct.FieldNames;
                        finalScore += compensater.Compensate(entity1Values,entity2Values);
                    }
                }

                this.lbl_CompareScore.Text = finalScore.ToString();
            }
        }

        void browser_OnClassSelected(string assemblyFilePath, string typeFullName)
        {
            if(!string.IsNullOrEmpty(this._CurrentSelectedFieldName))
            {
                if(this._EntityCollapseComponent.FieldComparers ==null)
                {
                    this._EntityCollapseComponent.FieldComparers=new Dictionary<string, FieldComparerPointer>();
                }

                FieldComparerPointer fieldComparer=new FieldComparerPointer();
                fieldComparer.AssemblyFileName = Path.GetFileName(assemblyFilePath);
                fieldComparer.TypeFullName = typeFullName;
                if(!this._EntityCollapseComponent.FieldComparers.ContainsKey(this._CurrentSelectedFieldName))
                {
                    this._EntityCollapseComponent.FieldComparers.Add(this._CurrentSelectedFieldName,fieldComparer);
                }
                else
                {
                    this._EntityCollapseComponent.FieldComparers[this._CurrentSelectedFieldName] = fieldComparer;
                }
            }
        }

        private void PopulateSchemas(string connStr)
        {
            this.cbo_Schemas.Items.Clear();
            this.cbo_Schemas.SelectedIndex = -1;
            this.cbo_Schemas.Text = string.Empty;

            string[] schemaNames = DbUtil.GetSchemaNames(DataProviderType.MSSQL, connStr);
            if(schemaNames !=null && schemaNames.Length>0)
            {
                foreach(string schemaName in schemaNames)
                {
                    this.cbo_Schemas.Items.Add(schemaName);
                }
            }
        }

        private void cbo_Schemas_SelectedIndexChanged(object sender, EventArgs e)
        {
            if(this.cbo_Schemas.SelectedIndex >=0 && (!string.IsNullOrEmpty(this.txt_ConnStr.Text)))
            {
                this.PopulateTableNames(this.txt_ConnStr.Text, this.cbo_Schemas.Text);
            }
        }

        private void PopulateTableNames(string connStr, string schemaName)
        {
            this.cbo_Tbls.Items.Clear();
            this.cbo_Tbls.SelectedIndex = -1;
            this.cbo_Tbls.Text = string.Empty;
            this.cbo_DistTable.Items.Clear();
            this.cbo_DistTable.SelectedIndex = -1;
            this.cbo_DistTable.Text = string.Empty;

            Dictionary<string, List<string>> schemaTables = DbUtil.GetTableNamesWithSchemas(DataProviderType.MSSQL, connStr);
            Dictionary<string, List<string>> schemaTables2=new Dictionary<string, List<string>>();
            foreach(string key in schemaTables.Keys)
            {
                schemaTables2.Add(key.ToLower(),schemaTables[key]);
            }
            schemaTables = schemaTables2;
            if(schemaTables !=null && schemaTables.ContainsKey(schemaName.ToLower()))
            {
                foreach(string tblName in schemaTables[schemaName.ToLower()])
                {
                    this.cbo_Tbls.Items.Add(tblName);
                    this.cbo_DistTable.Items.Add(tblName);
                }
            }
        }

        #region cluster
        private void btn_AddFilter_Click(object sender, EventArgs e)
        {
            List<string> addFilters=new List<string>();
            for(int i=0;i<this.clb_AddFilters.Items.Count;i++)
            {
                if(this.clb_AddFilters.GetItemChecked(i))
                {
                    addFilters.Add(this.clb_AddFilters.Items[i].ToString());
                }
            }
            string filterStr = string.Empty;
            for(int i=0;i<addFilters.Count;i++)
            {
                filterStr += addFilters[i];
                if(i<addFilters.Count -1)
                {
                    filterStr += " and ";
                }
            }
            this.lb_OrFilters.Items.Add(filterStr);

            List<string> distFilterFields=new List<string>();
            foreach(DataGridViewRow dr in this.dv_Encoders.Rows)
            {
                if(!dr.IsNewRow && dr.Cells[0].Value!=null)
                {
                    string fieldName = dr.Cells[0].Value.ToString();
                    if(!distFilterFields.Contains(fieldName))
                        distFilterFields.Add(fieldName);
                }
            }
            Assembly assembly = this.GetType().Assembly;
            List<string> encoderNames = new List<string>();
            Type[] types = assembly.GetTypes();
            foreach (Type type in types)
            {
                if (type.GetInterface(typeof(IFieldIndexer).FullName) != null)
                    encoderNames.Add(type.FullName);
            }
            foreach(string addField in addFilters)
            {
                if(!distFilterFields.Contains(addField))
                {
                    DataGridViewRow dr = new DataGridViewRow();
                    DataGridViewTextBoxCell cellFilter = new DataGridViewTextBoxCell();
                    cellFilter.Value = addField;
                    dr.Cells.Add(cellFilter);
                    DataGridViewComboBoxCell cellEncoders = new DataGridViewComboBoxCell();
                    if (encoderNames != null && encoderNames.Count > 0)
                    {
                        foreach (string encoderName in encoderNames)
                            cellEncoders.Items.Add(encoderName);
                    }
                    
                    dr.Cells.Add(cellEncoders);
                    this.dv_Encoders.Rows.Add(dr);
                }
            }
        }

        private void btn_RemoveFilter_Click(object sender, EventArgs e)
        {
            if(this.lb_OrFilters.SelectedIndex >=0)
            {
                string fieldToRemove = this.lb_OrFilters.SelectedItem.ToString();
                foreach(DataGridViewRow dr in this.dv_Encoders.Rows)
                {
                    if(!dr.IsNewRow && dr.Cells[0].Value!=null && 
                        dr.Cells[0].Value.ToString()==fieldToRemove)
                    {
                        this.dv_Encoders.Rows.Remove(dr);
                        break;
                    }
                }
                this.lb_OrFilters.Items.RemoveAt(this.lb_OrFilters.SelectedIndex);
            }
        }

        private void trk_SampleSize_Scroll(object sender, EventArgs e)
        {
            this.lbl_SampleSize.Text = string.Format("Test sample size: {0}", this.trk_SampleSize.Value);
        }
        #endregion

        #region swaps
        private void btn_AddSwapFields_Click(object sender, EventArgs e)
        {
            List<string> swapFields = new List<string>();
            for (int i = 0; i < this.clb_AvaiFields.Items.Count; i++)
            {
                if (this.clb_AvaiFields.GetItemChecked(i))
                {
                    swapFields.Add(this.clb_AvaiFields.Items[i].ToString());
                }
            }
            string swapFieldStr = string.Empty;
            for (int i = 0; i < swapFields.Count; i++)
            {
                swapFieldStr += swapFields[i];
                if (i < swapFields.Count - 1)
                {
                    swapFieldStr += " and ";
                }
            }
            this.lb_SwappedFields.Items.Add(swapFieldStr);
        }

        private void btn_RemoveSwappedFields_Click(object sender, EventArgs e)
        {
            if (this.lb_SwappedFields.SelectedIndex >=0)
            {
                this.lb_SwappedFields.Items.RemoveAt(this.lb_SwappedFields.SelectedIndex);
            }
        }
        #endregion

        #region compensate
        private void btn_AddCompositeFields_Click(object sender, EventArgs e)
        {
            for (int i = 0; i < this.clb_AvailableFields3.Items.Count; i++)
            {
                if (this.clb_AvailableFields3.GetItemChecked(i))
                {
                    this.lb_CompositeFields.Items.Add(this.clb_AvailableFields3.Items[i].ToString());
                }
            }
        }

        private void btn_RemoveCompositeFields_Click(object sender, EventArgs e)
        {
            if (this.lb_CompositeFields.SelectedIndex >= 0)
            {
                this.lb_CompositeFields.Items.RemoveAt(this.lb_CompositeFields.SelectedIndex);
            }
        }

        private void cbo_CompositeComparers_SelectedIndexChanged(object sender, EventArgs e)
        {
            if (this.cbo_CompositeComparers.SelectedIndex >= 0)
            {
                string compensateClassName = this.cbo_CompositeComparers.SelectedItem.ToString();
                if (compensateClassName == "Choose...")
                {
                }
            }
        }

        private void btn_AddCompositeComparer_Click(object sender, EventArgs e)
        {
            if (this.lb_CompositeFields.Items.Count > 0 && this.cbo_CompositeComparers.SelectedIndex > 0)
            {
                CompensaterPointer compensater = new CompensaterPointer();
                compensater.AssemblyName = Path.GetFileName(typeof(ICompensater).Assembly.Location);
                compensater.TypeFullName = this.cbo_CompositeComparers.SelectedItem.ToString();
                compensater.CanSwap = this.ck_CanSwapInCompositeFields.Checked;
                compensater.FieldNames = new List<string>();
                for (int i = 0; i < this.lb_CompositeFields.Items.Count; i++)
                {
                    compensater.FieldNames.Add(this.lb_CompositeFields.Items[i].ToString());
                }
                compensater.CompensateScore = (double)this.num_CompensateScore.Value;

                this.AddCompensater(compensater);
            }
        }

        private void AddCompensater(CompensaterPointer compensater)
        {
            DataGridViewRow dr = new DataGridViewRow();
            DataGridViewTextBoxCell cellAssemblyName = new DataGridViewTextBoxCell();
            cellAssemblyName.Value = compensater.AssemblyName;
            dr.Cells.Add(cellAssemblyName);
            DataGridViewTextBoxCell cellComparer = new DataGridViewTextBoxCell();
            cellComparer.Value = compensater.TypeFullName;
            dr.Cells.Add(cellComparer);
            DataGridViewCheckBoxCell cellSwap = new DataGridViewCheckBoxCell();
            cellSwap.Value = compensater.CanSwap;
            dr.Cells.Add(cellSwap);
            DataGridViewTextBoxCell cellFields = new DataGridViewTextBoxCell();
            cellFields.Value = StringUtil.ToString(compensater.FieldNames.ToArray());
            dr.Cells.Add(cellFields);
            DataGridViewTextBoxCell cellScore = new DataGridViewTextBoxCell();
            cellScore.Value = compensater.CompensateScore;
            dr.Cells.Add(cellScore);
            this.dv_CompositeComparer.Rows.Add(dr);
        }

        private List<CompensaterPointer> GetCompensaters()
        {
            List<CompensaterPointer> compensaters = new List<CompensaterPointer>();
            foreach (DataGridViewRow dr in this.dv_CompositeComparer.Rows)
            {
                if (!dr.IsNewRow && dr.Cells[0].Value != null && dr.Cells[1].Value != null)
                {
                    CompensaterPointer compensater = new CompensaterPointer();
                    compensater.AssemblyName =(string) dr.Cells[0].Value;
                    compensater.TypeFullName = (string)dr.Cells[1].Value;
                    compensater.CanSwap = (bool)dr.Cells[2].Value;
                    compensater.FieldNames = new List<string>();
                    compensater.FieldNames.AddRange(StringUtil.Split((string)dr.Cells[3].Value));
                    compensater.CompensateScore = double.Parse(dr.Cells[4].Value.ToString());
                    compensaters.Add(compensater);
                }
            }
            return compensaters;
        }
        #endregion

        private void dv_Fields_CellEndEdit(object sender, DataGridViewCellEventArgs e)
        {
            if (e.RowIndex >= 0 && e.ColumnIndex == 5)
            {
                DataGridViewRow dr = this.dv_Fields.Rows[e.RowIndex];
                if(dr.Cells[0].Value ==null || dr.Cells[6].Value==null)
                    return;

                DataGridViewTextBoxCell cellFieldName =
                    (DataGridViewTextBoxCell)dr.Cells[0];
                string currentFieldName = (string)cellFieldName.Value;
                string selectedTypeName = dr.Cells[6].Value.ToString();
                if (selectedTypeName == "...")
                    return;
                else if (selectedTypeName == "Choose...")
                {
                    this._CurrentSelectedFieldName = currentFieldName;
                    ClassBrowser browser = new ClassBrowser(this.GetType().Assembly, typeof(IAttributeComparer));
                    browser.OnClassSelected += new ClassSelected(browser_OnClassSelected);
                    if (this._EntityCollapseComponent.FieldComparers != null && this._EntityCollapseComponent.FieldComparers.ContainsKey(this._CurrentSelectedFieldName))
                    {
                        FieldComparerPointer fieldComparer =
                            this._EntityCollapseComponent.FieldComparers[this._CurrentSelectedFieldName];
                        if (!string.IsNullOrEmpty(fieldComparer.AssemblyFileName))
                        {
                            string binFolderPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
                            string assemblyFilePath = Path.Combine(binFolderPath, fieldComparer.AssemblyFileName);
                            browser.UpdateDisplay(assemblyFilePath, fieldComparer.TypeFullName);
                        }
                    }
                    browser.ShowDialog(this);
                }
                else if (this._SimpleTypeName_AssemblyName_Mappings.ContainsKey(selectedTypeName))
                {
                    FieldComparerPointer fieldComparer = new FieldComparerPointer();
                    fieldComparer.AssemblyFileName = this._SimpleTypeName_AssemblyName_Mappings[selectedTypeName];
                    fieldComparer.TypeFullName = this._SimpleTypeName_FullTypeName_Mappings[selectedTypeName];
                    if (this._EntityCollapseComponent.FieldComparers.ContainsKey(currentFieldName))
                    {
                        this._EntityCollapseComponent.FieldComparers[currentFieldName] = fieldComparer;
                    }
                    else
                    {
                        this._EntityCollapseComponent.FieldComparers.Add(currentFieldName, fieldComparer);
                    }
                }
            }
        }

        #region export to excel
        /// <summary>
        /// under collapse
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void exportToExcelToolStripMenuItem_Click(object sender, EventArgs e)
        {
            if (this._UnderCollapsedIDs != null && this._UnderCollapsedIDs.Count > 0)
            {
                try
                {
                    string collapsedIDFieldName = "CollapsedID";
                    List<string> colHeaders=new List<string>();
                    colHeaders.Add(collapsedIDFieldName);
                    colHeaders.Add(this._EntityCollapseComponent.IDFieldName);
                    foreach(string fieldName in this._EntityCollapseComponent.DescriptiveAttributes.Keys)
                    {
                        colHeaders.Add(fieldName);
                    }
                    foreach(string fieldName in this._EntityCollapseComponent.AssociativeAttributes.Keys)
                    {
                        colHeaders.Add(fieldName);
                    }
                    
                    // master 
                    DataTable dtMaster = DbUtil.GrabDataByKeyValues(
                        this._EntityCollapseComponent.ConnStr,
                        this._EntityCollapseComponent.CollapsedTableName,
                        collapsedIDFieldName,
                        this._UnderCollapsedIDs);
                    Dictionary<string,string> masterField_ColHeader_Mappings=new Dictionary<string, string>();
                    foreach(DataColumn col in dtMaster.Columns)
                    {
                        if(colHeaders.Contains(col.ColumnName))
                        {
                            masterField_ColHeader_Mappings.Add(col.ColumnName,col.ColumnName);
                        }
                    }

                    // association 
                    DataTable dtMappings = DbUtil.GrabDataByKeyValues(
                        this._EntityCollapseComponent.ConnStr,
                        this._EntityCollapseComponent.OutputTableName,
                        collapsedIDFieldName,
                        this._UnderCollapsedIDs);
                    List<int> entityIDs = new List<int>();
                    Dictionary<int, List<int>> collapseID_EntityID_Mappings = new Dictionary<int, List<int>>();
                    foreach (DataRow dr in dtMappings.Rows)
                    {
                        int entityID = (int)dr[this._EntityCollapseComponent.IDFieldName];
                        if(!entityIDs.Contains(entityID))
                        {
                            entityIDs.Add(entityID);
                        }
                        int collapseID = (int)dr[collapsedIDFieldName];
                        if(!collapseID_EntityID_Mappings.ContainsKey(collapseID))
                        {
                            List<int> detailIDs=new List<int>();
                            detailIDs.Add(entityID);
                            collapseID_EntityID_Mappings.Add(collapseID,detailIDs);
                        }
                        else
                        {
                            List<int> detailIDs = collapseID_EntityID_Mappings[collapseID];
                            if(!detailIDs.Contains(entityID))
                            {
                                detailIDs.Add(entityID);
                            }
                            collapseID_EntityID_Mappings[collapseID] = detailIDs;
                        }
                    }

                    // detail 
                    DataTable dtDetail = DbUtil.GrabDataByKeyValues(
                        this._EntityCollapseComponent.ConnStr,
                        this._EntityCollapseComponent.TableName,
                        this._EntityCollapseComponent.IDFieldName,
                        entityIDs);
                    Dictionary<string,string> detailField_ColHeader_Mappings=new Dictionary<string, string>();
                    foreach(DataColumn col in dtDetail.Columns)
                    {
                        if(colHeaders.Contains(col.ColumnName))
                        {
                            detailField_ColHeader_Mappings.Add(col.ColumnName,col.ColumnName);
                        }
                    }
                    ExcelTableWithGroupsUtil.FillTableWithGroup(
                        colHeaders, detailField_ColHeader_Mappings,
                        masterField_ColHeader_Mappings, collapsedIDFieldName,
                        this._EntityCollapseComponent.IDFieldName, 
                        collapseID_EntityID_Mappings,
                        dtMaster, dtDetail, false);

                }
                catch (Exception ex)
                {
                    MessageBox.Show(ex.ToString());
                }
            }
        }

        /// <summary>
        /// over collapse
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void exportToExcelToolStripMenuItem1_Click(object sender, EventArgs e)
        {
            if (this._OverCollapsedIDs != null && this._OverCollapsedIDs.Count > 0)
            {
                try
                {
                    // column headers
                    string collapsedIDFieldName = "CollapsedID";
                    List<string> colHeaders = new List<string>();
                    colHeaders.Add(collapsedIDFieldName);
                    colHeaders.Add(this._EntityCollapseComponent.IDFieldName);
                    foreach (string fieldName in this._EntityCollapseComponent.DescriptiveAttributes.Keys)
                    {
                        colHeaders.Add(fieldName);
                    }
                    foreach (string fieldName in this._EntityCollapseComponent.AssociativeAttributes.Keys)
                    {
                        colHeaders.Add(fieldName);
                    }

                    // master
                    DataTable dtMaster = DbUtil.GrabDataByKeyValues(
                        this._EntityCollapseComponent.ConnStr,
                        this._EntityCollapseComponent.CollapsedTableName,
                        collapsedIDFieldName,
                        this._OverCollapsedIDs);
                    Dictionary<string, string> masterField_ColHeader_Mappings = new Dictionary<string, string>();
                    foreach (DataColumn col in dtMaster.Columns)
                    {
                        if (colHeaders.Contains(col.ColumnName))
                        {
                            masterField_ColHeader_Mappings.Add(col.ColumnName, col.ColumnName);
                        }
                    }

                    // association
                    DataTable dtMappings = DbUtil.GrabDataByKeyValues(
                        this._EntityCollapseComponent.ConnStr,
                        this._EntityCollapseComponent.OutputTableName,
                        collapsedIDFieldName,
                        this._OverCollapsedIDs);
                    List<int> entityIDs = new List<int>();
                    Dictionary<int, List<int>> collapseID_EntityID_Mappings = new Dictionary<int, List<int>>();
                    foreach (DataRow dr in dtMappings.Rows)
                    {
                        int entityID = (int)dr[this._EntityCollapseComponent.IDFieldName];
                        if (!entityIDs.Contains(entityID))
                        {
                            entityIDs.Add(entityID);
                        }
                        int collapseID = (int)dr[collapsedIDFieldName];
                        if (!collapseID_EntityID_Mappings.ContainsKey(collapseID))
                        {
                            List<int> detailIDs = new List<int>();
                            detailIDs.Add(entityID);
                            collapseID_EntityID_Mappings.Add(collapseID, detailIDs);
                        }
                        else
                        {
                            List<int> detailIDs = collapseID_EntityID_Mappings[collapseID];
                            if (!detailIDs.Contains(entityID))
                            {
                                detailIDs.Add(entityID);
                            }
                            collapseID_EntityID_Mappings[collapseID] = detailIDs;
                        }
                    }

                    // detail
                    DataTable dtDetail = DbUtil.GrabDataByKeyValues(
                        this._EntityCollapseComponent.ConnStr,
                        this._EntityCollapseComponent.TableName,
                        this._EntityCollapseComponent.IDFieldName,
                        entityIDs);
                    Dictionary<string, string> detailField_ColHeader_Mappings = new Dictionary<string, string>();
                    foreach (DataColumn col in dtDetail.Columns)
                    {
                        if (colHeaders.Contains(col.ColumnName))
                        {
                            detailField_ColHeader_Mappings.Add(col.ColumnName, col.ColumnName);
                        }
                    }
                    ExcelTableWithGroupsUtil.FillTableWithGroup(
                        colHeaders, detailField_ColHeader_Mappings,
                        masterField_ColHeader_Mappings, collapsedIDFieldName,
                        this._EntityCollapseComponent.IDFieldName,
                        collapseID_EntityID_Mappings,
                        dtMaster, dtDetail, false);
                }
                catch (Exception ex)
                {
                    MessageBox.Show(ex.ToString());
                }
            }
        }
        
        private void btn_SaveOverCollapsed_Click(object sender, EventArgs e)
        {
            SaveFileDialog dlg = new SaveFileDialog();
            dlg.Filter = "Exce files(*.xlsx;*.xls)|*.xlsx;*.xls|All files(*.*)|*.*";
            if (dlg.ShowDialog() == DialogResult.OK)
            {
                this.txt_OverCollapsedFilePath.Text = dlg.FileName;
            }
        }

        private void btn_SaveUnderCollapsed_Click(object sender, EventArgs e)
        {
            SaveFileDialog dlg = new SaveFileDialog();
            dlg.Filter = "Exce files(*.xlsx;*.xls)|*.xlsx;*.xls|All files(*.*)|*.*";
            if (dlg.ShowDialog() == DialogResult.OK)
            {
                this.txt_UnderCollapsedFilePath.Text = dlg.FileName;
            }
        }

        private void ck_OverCollapsed_CheckedChanged(object sender, EventArgs e)
        {
            if (this.ck_OverCollapsed.Checked)
                this.txt_OverCollapsedFilePath.Enabled = true;
            else
                this.txt_OverCollapsedFilePath.Enabled = false;
        }

        private void ck_UnderCollapsed_CheckedChanged(object sender, EventArgs e)
        {
            if(this.ck_UnderCollapsed.Checked)
                this.txt_UnderCollapsedFilePath.Enabled = true;
            else
                this.txt_UnderCollapsedFilePath.Enabled = false;
        }

        private void btn_SaveFinalResult_Click(object sender, EventArgs e)
        {
            SaveFileDialog dlg = new SaveFileDialog();
            dlg.Filter = "Exce files(*.xlsx;*.xls)|*.xlsx;*.xls|All files(*.*)|*.*";
            if (dlg.ShowDialog() == DialogResult.OK)
            {
                this.txt_FinalResultFilePath.Text = dlg.FileName;
            }
        }

        private void ck_SaveFinalResult_CheckedChanged(object sender, EventArgs e)
        {
            if (this.ck_SaveFinalResult.Checked)
                this.txt_FinalResultFilePath.Enabled = true;
            else
                this.txt_FinalResultFilePath.Enabled = false;
        }
        #endregion

        private void btn_CompareField_Click(object sender, EventArgs e)
        {
            if(this.cbo_Comparers.SelectedIndex>=0)
            {
                try
                {
                    Assembly assembly = this.GetType().Assembly;
                    Type type = assembly.GetType(this.cbo_Comparers.SelectedItem.ToString());
                    IAttributeComparer comparer = (IAttributeComparer) Activator.CreateInstance(type);
                    SimpleDataType dbType = SimpleDataTypeUtil.ToSimpleDataType(comparer.FieldDbType);
                    object value1 = SimpleDataTypeUtil.ToDbValue(dbType, this.txt_FieldValue1.Text);
                    object value2 = SimpleDataTypeUtil.ToDbValue(dbType, this.txt_FieldValue2.Text);
                    double score = comparer.Compare(value1, value2, 0, null, null);
                    this.lbl_FieldCompareScore.Text = string.Format("Score: {0}", Math.Round(score, 2));
                }
                catch(Exception ex)
                {
                    MessageBox.Show(ex.Message);
                }
            }
        }


    }
}
